[ { "memory": 3077064960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 1 } }, { "memory": 3086272768, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 1, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3088107776, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 1, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3089942784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 1, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3091777792, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 1, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3092138240, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 1, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3093973248, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 1, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3095808256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 1, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3097643264, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 1, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3098003712, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 1, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3107211520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 1, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3116419328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 1, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3118254336, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 1, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3127462144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 1, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3129297152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 1, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3129657600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 1, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3130018048, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 1, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3139225856, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 1, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3148433664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 1, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3150268672, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 1, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3150629120, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 1, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3150989568, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 1, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3152824576, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 1, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3162032384, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 1, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3162392832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 1, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3162753280, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 1, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3171961088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 1, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3173796096, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 1, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3174156544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 1, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3174516992, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 1, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3176352000, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 1, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3176712448, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 1, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3178547456, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 1, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3187755264, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 1, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3188115712, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 1, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3197323520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 1, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3206531328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 1, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3215739136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 1, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3224946944, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 1, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3234154752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 1, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3235989760, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 1, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3236350208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 1, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3245558016, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 1, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3254765824, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 1, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3263973632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 1, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3273181440, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 1, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3273541888, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 1, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3275376896, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 1, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3277211904, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 1, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3286419712, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 1, "layer.27.mlp.down_proj": 2 } }, { "memory": 3295627520, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 1, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3304835328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 1, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3305195776, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 1, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3307030784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 1, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3316238592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 1, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3316599040, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 1, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3316959488, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 1, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3326167296, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 1, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3335375104, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 1, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3344582912, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 1, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3353790720, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 1, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3354151168, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 1, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3363358976, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 1, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3372566784, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 1, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3381774592, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 1, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3390982400, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 1, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3400190208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 1, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3409398016, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 1, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3411233024, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 1, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3420440832, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 1, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3429648640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 1, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3438856448, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 1, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3448064256, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 1, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3457272064, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 1, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3457632512, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 1, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3466840320, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 1, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3468675328, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 1, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3477883136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 1, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3479718144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 1, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3481553152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 1, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3481913600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 1, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3483748608, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 1, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3492956416, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 1, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3494791424, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 1, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3495151872, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 1, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3504359680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 1, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3506194688, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 1, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3508029696, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 1, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3508390144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 1, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3510225152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 1, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3510585600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 1, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3510946048, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 1, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3512781056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 1, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3513141504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 1, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3513501952, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 1, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3515336960, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 1, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3515697408, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 1, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3516057856, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 1, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3517892864, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 1, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3518253312, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 1, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3518613760, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 1, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3518974208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 1, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3520809216, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 1, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3521169664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 1, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3521530112, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 1, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3523365120, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 1, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3525200128, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 1, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3525560576, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 1, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3525921024, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 1, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3527756032, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 1, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3536963840, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 1, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3537324288, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 1, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3537684736, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 1, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3539519744, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 1, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3541354752, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 1, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3541715200, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 1, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3543550208, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 1, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3543910656, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 1, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3545745664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 1, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3547580672, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 1, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3549415680, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 1, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3551250688, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 1, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3551611136, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 1, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3553446144, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 1, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3555281152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 1, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3555641600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 1, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3557476608, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 1, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3559311616, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 1, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3561146624, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 1, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3562981632, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 1, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3564816640, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 1, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3565177088, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 1, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3565537536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 1, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3574745344, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 1, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3583953152, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 1, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3584313600, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 1, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3593521408, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 1, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3593881856, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 1, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3603089664, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 1, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3612297472, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 1, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3621505280, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 1, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3621865728, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 1, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3631073536, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 1, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3632908544, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 1, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3642116352, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 1, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3642476800, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 1, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3642837248, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 1, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3652045056, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 1, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3652405504, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 1, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3652765952, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 1, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3661973760, "layers": { "layer.0.self_attn.q_proj": 1, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3663808768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 1, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3673016576, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 1, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3674851584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 1, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3684059392, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 1, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3693267200, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 1, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3695102208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 1, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3704310016, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 1, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3713517824, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 1, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3722725632, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 1, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3731933440, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 1, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3741141248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 1, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3750349056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 1, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3750709504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 1, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3759917312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 1, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3760277760, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 1, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3769485568, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 1, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3778693376, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 1, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3787901184, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 1, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3797108992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 1, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3806316800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 1, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3815524608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 1, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3815885056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 1, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3816245504, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 1, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3825453312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 1, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3827288320, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 1, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3836496128, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 1, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3845703936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 1, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3854911744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 1, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3864119552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 1, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3873327360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 1, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3875162368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 1, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3884370176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 1, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3893577984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 1, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3902785792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 1, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3904620800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 1, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3906455808, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 1, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3906816256, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 1, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3916024064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 1, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3925231872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 1, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3934439680, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 1, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3943647488, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 1, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3944007936, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 1, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3953215744, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 1, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3955050752, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 1, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3964258560, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 1, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3973466368, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 2, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3982674176, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 2, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3983034624, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 2, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 3992242432, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 2, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 4001450240, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 2 } }, { "memory": 4010658048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 2, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4012493056, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 2, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4014328064, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 2, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4023535872, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 2, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4025370880, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 2, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4034578688, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 2, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4043786496, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 2, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4052994304, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 2, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4062202112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 2, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4071409920, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 2, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4080617728, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 2, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4089825536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 2, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4090185984, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 2, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4099393792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 2, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4101228800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 2, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4110436608, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 2, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4119644416, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 2, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4120004864, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 2, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4120365312, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 2, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4129573120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 2, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4138780928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 2, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4147988736, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 2, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4157196544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 2, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4166404352, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 2, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4175612160, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 2, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4184819968, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 2, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4186654976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 2, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4195862784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 2, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4205070592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 2, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4206905600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 2, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4216113408, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 2, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4225321216, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 2, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4234529024, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 2, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4243736832, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 2, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4244097280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 2, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4245932288, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 2, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4255140096, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 2, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4255500544, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 2, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4255860992, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 2, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4265068800, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 2, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4265429248, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 2, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4265789696, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 2, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4267624704, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 2, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4276832512, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 2, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4277192960, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 2, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4286400768, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 2, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4288235776, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 2, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4297443584, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 2, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4297804032, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 2, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4299639040, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 2, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4308846848, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 2, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4309207296, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 2, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4318415104, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 2, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4318775552, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 2, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4327983360, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 2, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4337191168, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 2, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4346398976, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 2, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4355606784, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 2, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4364814592, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 2, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4374022400, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 2, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4383230208, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 2, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4383590656, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 2, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4392798464, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 2, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4394633472, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 2, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4403841280, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 2, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4413049088, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 2, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4413409536, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 2, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4422617344, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 2, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4422977792, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 2, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4432185600, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 2, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4432546048, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 2, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4441753856, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 2, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4450961664, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 2, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4451322112, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 2, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4453157120, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 2, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4462364928, "layers": { "layer.0.self_attn.q_proj": 2, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4464199936, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 2, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4473407744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 2, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4473768192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 2, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4475603200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 2, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4477438208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 2, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4486646016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 2, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4488481024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 2, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4497688832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 2, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4498049280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 2, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4499884288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 2, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4500244736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 2, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4502079744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 2, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4511287552, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 2, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4513122560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 2, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4522330368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 2, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4522690816, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 2, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4524525824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 2, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4526360832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 2, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4528195840, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 2, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4537403648, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 2, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4546611456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 2, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4555819264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 2, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4565027072, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 2, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4566862080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 2, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4567222528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 2, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4569057536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 2, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4570892544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 2, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4571252992, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 2, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4573088000, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 2, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4574923008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 2, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4575283456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 2, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4577118464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 2, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4577478912, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 2, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4577839360, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 2, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4579674368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 2, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4588882176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 2, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4590717184, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 2, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4592552192, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 2, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4592912640, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 2, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4593273088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 2, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4593633536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 2, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4593993984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 2, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4594354432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 2, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4603562240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 2, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4612770048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 2, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4621977856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 2, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4623812864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 2, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4633020672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 2, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4642228480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 2, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4644063488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 2, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4645898496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 2, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4646258944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 2, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4648093952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 2, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4648454400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 2, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4648814848, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 2, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4649175296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 2, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4651010304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 2, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4651370752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 2, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4653205760, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 2, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4662413568, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 2, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4671621376, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 2, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4671981824, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 2, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4681189632, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 2, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4690397440, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 2, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4690757888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 2, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4692592896, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 2, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4692953344, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 2, "layer.27.mlp.down_proj": 3 } }, { "memory": 4702161152, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 2, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4702521600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 2, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4711729408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 2, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4712089856, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 2, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4713924864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 2, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4723132672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 2, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4724967680, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 2, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4726802688, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 2, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4727163136, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 2, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4736370944, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 2, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4738205952, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 2, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4740040960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 2, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4741875968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 2, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4742236416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 2, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4742596864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 2, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4744431872, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 2, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4744792320, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 2, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4754000128, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 2, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4754360576, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 2, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4756195584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 2, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4756556032, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 2, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4758391040, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 2, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4758751488, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 2, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4767959296, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 2, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4768319744, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 2, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4770154752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 2, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4779362560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 2, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4779723008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 2, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4780083456, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 2, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4789291264, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 2, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4791126272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 2, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4800334080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 2, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4802169088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 2, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4802529536, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 2, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4802889984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 2, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4812097792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 2, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4813932800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 2, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4823140608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 2, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4832348416, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 2, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4834183424, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 2, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4836018432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 2, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4845226240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 2, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4847061248, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 2, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4856269056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 2, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4858104064, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 2, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4858464512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 2, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4858824960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 2, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4868032768, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 2, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4869867776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 3 } }, { "memory": 4879075584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 3, "layer.27.mlp.down_proj": 4 } }, { "memory": 4888283392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 3, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4890118400, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 3, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4899326208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 3, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4908534016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 3, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4908894464, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 3, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4918102272, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 3, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4927310080, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 3, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4936517888, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 3, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4945725696, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 3, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4954933504, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 3, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4964141312, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 3, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4973349120, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 3, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4982556928, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 3, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 4991764736, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 3, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5000972544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 3, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5010180352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 3, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5019388160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 3, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5028595968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 3, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5037803776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 3, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5047011584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 3, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5056219392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 3, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5065427200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 3, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5074635008, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 3, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5083842816, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 3, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5093050624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 3, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5102258432, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 3, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5111466240, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 3, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5120674048, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 3, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5121034496, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 3, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5130242304, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 3, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5130602752, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 3, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5139810560, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 3, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5149018368, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 3, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5158226176, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 3, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5167433984, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 3, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5176641792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 3, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5185849600, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 3, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5187684608, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 3, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5188045056, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 3, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5197252864, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 3, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5206460672, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 3, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5215668480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 3, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5224876288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 3, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5234084096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 3, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5234444544, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 3, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5243652352, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 3, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5252860160, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 3, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5262067968, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 3, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5271275776, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 3, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5280483584, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 3, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5289691392, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 3, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5298899200, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 3, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5300734208, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 3, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5309942016, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 3, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5311777024, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 3, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5320984832, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 3, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5321345280, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 3, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5330553088, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 3, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5339760896, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 3, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5348968704, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 3, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5358176512, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 3, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5358536960, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 3, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5358897408, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 3, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5368105216, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 3, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5368465664, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 3, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5377673472, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 3, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5379508480, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 3, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5388716288, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 3, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5397924096, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 3, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5407131904, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 3, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5416339712, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 3, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5418174720, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 3, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5427382528, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 3, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5436590336, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 3, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5436950784, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 3, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5438785792, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 3, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5440620800, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 3, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5442455808, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 3, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5451663616, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 3, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5453498624, "layers": { "layer.0.self_attn.q_proj": 3, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5455333632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 3, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5464541440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 3, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5473749248, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 3, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5482957056, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 3, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5483317504, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 3, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5485152512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 3, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5485512960, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 3, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5487347968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 3, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5487708416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 3, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5496916224, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 3, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5506124032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 3, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5515331840, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 3, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5515692288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 3, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5517527296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 3, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5526735104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 3, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5535942912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 3, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5545150720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 3, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5554358528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 3, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5556193536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 3, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5556553984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 3, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5565761792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 3, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5574969600, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 3, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5575330048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 3, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5584537856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 3, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5584898304, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 3, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5586733312, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 3, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5587093760, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 3, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5596301568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 3, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5605509376, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 3, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5607344384, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 3, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5607704832, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 3, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5608065280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 3, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5609900288, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 3, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5619108096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 3, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5620943104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 3, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5630150912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 3, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5630511360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 3, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5632346368, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 3, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5632706816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 3, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5633067264, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 3, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5634902272, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 3, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5636737280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 3, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5637097728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 3, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5638932736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 3, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5639293184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 3, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5639653632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 3, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5640014080, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 3, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5641849088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 3, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5643684096, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 3, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5652891904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 3, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5654726912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 3, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5656561920, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 3, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5665769728, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 3, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5666130176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 3, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5666490624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 3, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5666851072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 3, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5667211520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 3, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5669046528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 3, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5670881536, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 3, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5671241984, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 3, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5671602432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 3, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5671962880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 3, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5672323328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 3, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5672683776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 3, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5681891584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 3, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5682252032, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 3, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5684087040, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 3, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5693294848, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 3, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5693655296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 3, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5702863104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 3, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5704698112, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 3, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5706533120, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 3, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5706893568, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 3, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5707254016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 3, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5709089024, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 3, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5709449472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 3, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5711284480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 3, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5711644928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 3, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5713479936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 3, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5715314944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 3, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5717149952, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 3, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5717510400, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 3, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5719345408, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 3, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5721180416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 3, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5721540864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 3, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5723375872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 3, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5723736320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 3, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5724096768, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 3, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5725931776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 3, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5727766784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 3, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5729601792, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 3, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5731436800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 3, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5733271808, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 3, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5733632256, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 3, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5733992704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 3, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5735827712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 3, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5737662720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 3, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5738023168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 3, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5738383616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 3, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5740218624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 3, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5740579072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 3, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5740939520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 3, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5742774528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 3, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5743134976, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 3, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5743495424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 3, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5743855872, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 3, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5745690880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 3, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5746051328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 3, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5747886336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 3, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5749721344, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 3, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5751556352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 3, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5753391360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 3, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5762599168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 3, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5764434176, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 3, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5766269184, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 4 } }, { "memory": 5775476992, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 4, "layer.27.mlp.down_proj": 5 } }, { "memory": 5784684800, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 4, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5793892608, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 4, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5803100416, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 4, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5803460864, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 4, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5812668672, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 4, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5821876480, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 4, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5822236928, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 4, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5831444736, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 4, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5840652544, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 4, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5849860352, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 4, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5859068160, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 4, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5868275968, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 4, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5877483776, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 4, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5886691584, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 4, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5895899392, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 4, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5905107200, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 4, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5914315008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 4, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5923522816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 4, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5932730624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 4, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5941938432, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 4, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5951146240, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 4, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5960354048, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 4, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5969561856, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 4, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5978769664, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 4, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5987977472, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 4, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 5997185280, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 4, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6006393088, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 4, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6015600896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 4, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6024808704, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 4, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6034016512, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 4, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6043224320, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 4, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6052432128, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 4, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6061639936, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 4, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6070847744, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 4, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6080055552, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 4, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6089263360, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 4, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6098471168, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 4, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6098831616, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 4, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6108039424, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 4, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6117247232, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 4, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6117607680, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 4, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6126815488, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 4, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6136023296, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 4, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6145231104, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 4, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6154438912, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 4, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6163646720, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 4, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6172854528, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 4, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6182062336, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 4, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6182422784, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 4, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6191630592, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 4, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6200838400, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 4, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6210046208, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 4, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6219254016, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 4, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6228461824, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 4, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6237669632, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 4, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6246877440, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 4, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6247237888, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 4, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6249072896, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 4, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6250907904, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 4, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6260115712, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 4, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6269323520, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 4, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6278531328, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 4, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6287739136, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 4, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6296946944, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 4, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6306154752, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 4, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6315362560, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 4, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6315723008, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 4, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6324930816, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 4, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6334138624, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 4, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6334499072, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 4, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6343706880, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 4, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6352914688, "layers": { "layer.0.self_attn.q_proj": 4, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6354749696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 4, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6356584704, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 4, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6365792512, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 4, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6367627520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 4, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6369462528, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 4, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6378670336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 4, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6380505344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 4, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6389713152, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 4, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6398920960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 4, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6408128768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 4, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6409963776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 4, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6410324224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 4, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6419532032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 4, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6419892480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 4, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6421727488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 4, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6423562496, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 4, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6432770304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 4, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6441978112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 4, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6451185920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 4, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6453020928, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 4, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6453381376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 4, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6455216384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 4, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6464424192, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 4, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6464784640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 4, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6465145088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 4, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6465505536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 4, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6465865984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 4, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6466226432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 4, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6466586880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 4, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6468421888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 4, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6477629696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 4, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6486837504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 4, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6487197952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 4, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6487558400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 4, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6489393408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 4, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6489753856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 4, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6491588864, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 4, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6491949312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 4, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6492309760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 4, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6492670208, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 4, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6494505216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 4, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6503713024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 4, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6504073472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 4, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6513281280, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 4, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6513641728, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 4, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6514002176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 4, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6514362624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 4, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6514723072, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 4, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6523930880, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 4, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6524291328, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 4, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6526126336, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 4, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6527961344, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 4, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6529796352, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 4, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6530156800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 4, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6530517248, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 4, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6539725056, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 4, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6548932864, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 4, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6558140672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 4, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6558501120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 4, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6560336128, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 4, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6569543936, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 4, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6571378944, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 4, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6573213952, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 4, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6575048960, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 4, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6575409408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 4, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6575769856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 4, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6576130304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 4, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6576490752, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 4, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6578325760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 4, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6578686208, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 4, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6579046656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 4, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6588254464, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 4, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6588614912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 4, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6588975360, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 4, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6590810368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 4, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6592645376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 4, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6594480384, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 4, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6594840832, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 4, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6596675840, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 4, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6597036288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 4, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6598871296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 4, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6600706304, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 4, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6602541312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 4, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6602901760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 4, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6604736768, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 4, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6606571776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 4, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6606932224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 4, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6607292672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 4, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6609127680, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 4, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6610962688, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 4, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6611323136, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 4, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6611683584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 4, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6613518592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 4, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6615353600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 4, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6617188608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 4, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6619023616, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 4, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6620858624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 4, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6622693632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 4, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6624528640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 4, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6624889088, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 4, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6625249536, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 4, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6627084544, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 4, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6628919552, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 4, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6629280000, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 4, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6631115008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 4, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6632950016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 4, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6634785024, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 4, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6635145472, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 4, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6635505920, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 4, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6635866368, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 4, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6645074176, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 4, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6645434624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 4, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6647269632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 4, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6649104640, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 4, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6650939648, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 4, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6652774656, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 4, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6653135104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 4, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6654970112, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 4, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6656805120, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 4, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6657165568, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 4, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6659000576, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 4, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6660835584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 4, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6662670592, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 5 } }, { "memory": 6671878400, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 5, "layer.27.mlp.down_proj": 6 } }, { "memory": 6681086208, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 5, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6690294016, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 5, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6699501824, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 5, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6708709632, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 5, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6717917440, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 5, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6718277888, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 5, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6727485696, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 5, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6736693504, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 5, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6745901312, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 5, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6746261760, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 5, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6755469568, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 5, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6764677376, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 5, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6773885184, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 5, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6783092992, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 5, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6792300800, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 5, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6801508608, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 5, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6810716416, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 5, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6819924224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 5, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6820284672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 5, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6829492480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 5, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6838700288, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 5, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6847908096, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 5, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6857115904, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 5, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6866323712, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 5, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6875531520, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 5, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6875891968, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 5, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6885099776, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 5, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6894307584, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 5, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6903515392, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 5, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6912723200, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 5, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6921931008, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 5, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6931138816, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 5, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6940346624, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 5, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6949554432, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 5, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6958762240, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 5, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6967970048, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 5, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6977177856, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 5, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6979012864, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 5, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6988220672, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 5, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6997428480, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 5, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 6999263488, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 5, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7008471296, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 5, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7017679104, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 5, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7026886912, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 5, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7036094720, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 5, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7036455168, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 5, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7045662976, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 5, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7047497984, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 5, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7056705792, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 5, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7065913600, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 5, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7075121408, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 5, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7084329216, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 5, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7086164224, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 5, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7095372032, "layers": { "layer.0.self_attn.q_proj": 5, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7097207040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 5, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7106414848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 5, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7106775296, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 5, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7115983104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 5, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7125190912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 5, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7134398720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 5, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7143606528, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 5, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7143966976, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 5, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7145801984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 5, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7155009792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 5, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7155370240, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 5, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7164578048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 5, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7173785856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 5, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7182993664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 5, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7183354112, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 5, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7183714560, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 5, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7192922368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 5, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7202130176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 5, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7211337984, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 5, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7220545792, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 5, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7229753600, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 5, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7230114048, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 5, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7230474496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 5, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7232309504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 5, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7234144512, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 5, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7234504960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 5, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7243712768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 5, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7245547776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 5, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7245908224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 5, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7246268672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 5, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7255476480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 5, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7255836928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 5, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7257671936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 5, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7259506944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 5, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7261341952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 5, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7261702400, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 5, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7263537408, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 5, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7263897856, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 5, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7273105664, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 5, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7282313472, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 5, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7282673920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 5, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7283034368, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 5, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7284869376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 5, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7286704384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 5, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7288539392, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 5, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7288899840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 5, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7298107648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 5, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7307315456, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 5, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7307675904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 5, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7316883712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 5, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7317244160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 5, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7317604608, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 5, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7326812416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 5, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7336020224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 5, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7345228032, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 5, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7354435840, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 5, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7363643648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 5, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7365478656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 5, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7365839104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 5, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7367674112, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 5, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7369509120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 5, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7378716928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 5, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7380551936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 5, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7380912384, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 5, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7381272832, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 5, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7381633280, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 5, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7390841088, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 5, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7392676096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 5, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7393036544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 5, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7393396992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 5, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7395232000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 5, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7395592448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 5, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7395952896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 5, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7396313344, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 5, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7398148352, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 5, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7398508800, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 5, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7400343808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 5, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7400704256, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 5, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7401064704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 5, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7402899712, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 5, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7403260160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 5, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7412467968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 5, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7412828416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 5, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7422036224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 5, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7423871232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 5, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7433079040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 5, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7433439488, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 5, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7435274496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 5, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7435634944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 5, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7444842752, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 5, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7445203200, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 5, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7445563648, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 5, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7445924096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 5, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7446284544, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 5, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7446644992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 5, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7447005440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 5, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7448840448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 5, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7449200896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 5, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7451035904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 5, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7452870912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 5, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7454705920, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 5, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7456540928, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 5, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7458375936, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 5, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7467583744, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 5, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7469418752, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 5, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7471253760, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 5, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7473088768, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 5, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7474923776, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 5, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7475284224, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 5, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7475644672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 5, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7477479680, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 5, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7479314688, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 5, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7488522496, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 5, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7488882944, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 5, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7490717952, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 5, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7492552960, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 5, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7494387968, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 5, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7494748416, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 5, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7496583424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 5, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7496943872, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 5, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7498778880, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 5, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7500613888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 5, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7502448896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 5, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7504283904, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 5, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7506118912, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 5, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7515326720, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 5, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7515687168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 5, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7517522176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 5, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7519357184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 5, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7528564992, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 5, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7530400000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 5, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7530760448, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 5, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7531120896, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 5, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7540328704, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 5, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7540689152, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 5, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7542524160, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 5, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7544359168, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 5, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7546194176, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 5, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7548029184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 5, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7549864192, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 5, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7559072000, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 6 } }, { "memory": 7568279808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 6, "layer.27.mlp.down_proj": 7 } }, { "memory": 7577487616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 6, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7586695424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 6, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7595903232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 6, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7605111040, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 6, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7614318848, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 6, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7623526656, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 6, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7632734464, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 6, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7641942272, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 6, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7651150080, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 6, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7660357888, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 6, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7669565696, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 6, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7678773504, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 6, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7687981312, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 6, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7697189120, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 6, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7697549568, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 6, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7706757376, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 6, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7715965184, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 6, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7716325632, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 6, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7725533440, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 6, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7734741248, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 6, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7743949056, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 6, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7753156864, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 6, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7762364672, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 6, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7771572480, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 6, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7780780288, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 6, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7789988096, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 6, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7791823104, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 6, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7792183552, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 6, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7801391360, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 6, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7801751808, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 6, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7810959616, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 6, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7820167424, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 6, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7829375232, "layers": { "layer.0.self_attn.q_proj": 6, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7831210240, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 6, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7831570688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 6, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7840778496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 6, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7849986304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 6, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7851821312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 6, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7853656320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 6, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7862864128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 6, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7872071936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 6, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7872432384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 6, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7881640192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 6, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7890848000, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 6, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7900055808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 6, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7909263616, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 6, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7918471424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 6, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7927679232, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 6, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7936887040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 6, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7946094848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 6, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7955302656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 6, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7957137664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 6, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7957498112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 6, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7957858560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 6, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7967066368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 6, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7967426816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 6, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7976634624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 6, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7985842432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 6, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7986202880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 6, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 7995410688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 6, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8004618496, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 6, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8013826304, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 6, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8023034112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 6, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8023394560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 6, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8023755008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 6, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8032962816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 6, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8034797824, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 6, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8036632832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 6, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8038467840, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 6, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8038828288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 6, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8048036096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 6, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8049871104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 6, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8051706112, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 6, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8052066560, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 6, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8052427008, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 6, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8061634816, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 6, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8070842624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 6, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8080050432, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 6, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8081885440, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 6, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8083720448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 6, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8085555456, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 6, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8094763264, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 6, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8096598272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 6, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8098433280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 6, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8100268288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 6, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8109476096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 6, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8111311104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 6, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8111671552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 6, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8112032000, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 6, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8121239808, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 6, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8121600256, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 6, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8121960704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 6, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8122321152, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 6, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8122681600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 6, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8131889408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 6, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8141097216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 6, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8150305024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 6, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8150665472, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 6, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8152500480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 6, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8161708288, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 6, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8162068736, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 6, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8162429184, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 6, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8164264192, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 6, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8164624640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 6, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8173832448, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 6, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8174192896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 6, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8174553344, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 6, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8174913792, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 6, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8176748800, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 6, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8185956608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 6, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8186317056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 6, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8188152064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 6, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8189987072, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 6, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8199194880, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 6, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8208402688, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 6, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8210237696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 6, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8219445504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 6, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8221280512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 6, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8223115520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 6, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8223475968, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 6, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8225310976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 6, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8225671424, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 6, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8226031872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 6, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8226392320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 6, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8226752768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 6, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8235960576, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 6, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8245168384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 6, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8245528832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 6, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8254736640, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 6, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8256571648, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 6, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8256932096, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 6, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8258767104, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 6, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8259127552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 6, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8268335360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 6, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8270170368, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 6, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8279378176, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 6, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8279738624, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 6, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8281573632, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 6, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8281934080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 6, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8282294528, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 6, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8291502336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 6, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8291862784, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 6, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8301070592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 6, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8302905600, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 6, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8304740608, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 6, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8305101056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 6, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8305461504, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 6, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8314669312, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 6, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8316504320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 6, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8325712128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 6, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8326072576, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 6, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8327907584, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 6, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8328268032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 6, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8330103040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 6, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8331938048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 6, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8333773056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 6, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8335608064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 6, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8344815872, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 6, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8345176320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 6, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8345536768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 6, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8345897216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 6, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8355105024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 6, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8356940032, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 6, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8358775040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 6, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8360610048, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 6, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8362445056, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 6, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8364280064, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 6, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8364640512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 6, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8366475520, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 6, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8375683328, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 6, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8377518336, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 6, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8386726144, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 6, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8387086592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 6, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8387447040, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 6, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8396654848, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 6, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8405862656, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 6, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8407697664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 6, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8409532672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 6, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8411367680, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 6, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8411728128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 6, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8413563136, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 6, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8422770944, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 6, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8424605952, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 6, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8426440960, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 6, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8435648768, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 6, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8436009216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 6, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8436369664, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 6, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8438204672, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 6, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8447412480, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 6, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8449247488, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 6, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8449607936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 6, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8451442944, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 6, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8451803392, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 6, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8453638400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 6, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8455473408, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 7 } }, { "memory": 8464681216, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 7, "layer.27.mlp.down_proj": 8 } }, { "memory": 8473889024, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 7, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8483096832, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 7, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8483457280, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 7, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8492665088, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 7, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8501872896, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 7, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8511080704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 7, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8520288512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 7, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8529496320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 7, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8538704128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 7, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8547911936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 7, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8557119744, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 7, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8566327552, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 7, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8575535360, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 7, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8584743168, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 7, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8593950976, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 7, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8603158784, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 7, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8612366592, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 7, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8621574400, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 7, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8630782208, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 7, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8639990016, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 7, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8640350464, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 7, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8649558272, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 7, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8658766080, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 7, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8667973888, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 7, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8677181696, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 7, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8679016704, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 7, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8688224512, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 7, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8697432320, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 7, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8706640128, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 7, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8715847936, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 7, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8716208384, "layers": { "layer.0.self_attn.q_proj": 7, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8718043392, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 7, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8727251200, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 7, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8736459008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 7, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8745666816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 7, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8746027264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 7, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8755235072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 7, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8764442880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 7, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8773650688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 7, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8782858496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 7, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8792066304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 7, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8801274112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 7, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8810481920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 7, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8819689728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 7, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8828897536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 7, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8838105344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 7, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8847313152, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 7, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8856520960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 7, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8858355968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 7, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8867563776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 7, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8869398784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 7, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8878606592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 7, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8880441600, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 7, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8889649408, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 7, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8890009856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 7, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8899217664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 7, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8901052672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 7, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8910260480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 7, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8910620928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 7, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8910981376, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 7, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8920189184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 7, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8920549632, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 7, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8929757440, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 7, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8938965248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 7, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8940800256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 7, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8950008064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 7, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8950368512, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 7, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8959576320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 7, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8961411328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 7, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8961771776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 7, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8963606784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 7, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8963967232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 7, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8973175040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 7, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8982382848, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 7, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8984217856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 7, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8986052864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 7, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 8995260672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 7, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9004468480, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 7, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9013676288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 7, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9014036736, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 7, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9014397184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 7, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9023604992, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 7, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9032812800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 7, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9042020608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 7, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9043855616, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 7, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9044216064, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 7, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9053423872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 7, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9053784320, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 7, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9054144768, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 7, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9054505216, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 7, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9054865664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 7, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9055226112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 7, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9055586560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 7, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9055947008, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 7, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9056307456, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 7, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9065515264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 7, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9074723072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 7, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9083930880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 7, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9085765888, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 7, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9094973696, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 7, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9096808704, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 7, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9098643712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 7, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9099004160, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 7, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9099364608, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 7, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9099725056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 7, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9108932864, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 7, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9118140672, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 7, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9118501120, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 7, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9127708928, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 7, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9129543936, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 7, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9129904384, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 7, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9130264832, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 7, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9139472640, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 7, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9139833088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 7, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9141668096, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 7, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9150875904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 7, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9160083712, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 7, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9161918720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 7, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9162279168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 7, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9171486976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 7, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9171847424, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 7, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9172207872, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 7, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9174042880, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 7, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9174403328, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 7, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9174763776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 7, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9176598784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 7, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9176959232, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 7, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9177319680, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 7, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9177680128, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 7, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9179515136, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 7, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9181350144, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 7, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9181710592, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 7, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9182071040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 7, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9183906048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 7, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9184266496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 7, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9186101504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 7, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9186461952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 7, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9188296960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 7, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9190131968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 7, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9199339776, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 7, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9208547584, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 7, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9208908032, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 7, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9210743040, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 7, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9212578048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 7, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9214413056, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 7, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9214773504, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 7, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9215133952, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 7, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9216968960, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 7, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9218803968, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 7, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9220638976, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 7, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9229846784, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 7, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9231681792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 7, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9233516800, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 7, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9233877248, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 7, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9235712256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 7, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9237547264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 7, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9239382272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 7, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9241217280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 7, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9243052288, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 7, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9244887296, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 7, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9245247744, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 7, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9254455552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 7, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9263663360, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 7, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9265498368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 7, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9274706176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 7, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9276541184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 7, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9278376192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 7, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9287584000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 7, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9287944448, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 7, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9288304896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 7, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9290139904, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 7, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9291974912, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 7, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9293809920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 7, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9303017728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 7, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9303378176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 7, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9305213184, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 7, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9307048192, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 7, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9316256000, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 7, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9325463808, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 7, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9325824256, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 7, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9327659264, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 7, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9329494272, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 7, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9329854720, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 7, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9330215168, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 7, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9332050176, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 7, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9332410624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 7, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9332771072, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 7, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9334606080, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 7, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9334966528, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 7, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9336801536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 7, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9338636544, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 7, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9340471552, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 7, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9342306560, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 7, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9351514368, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 7, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9351874816, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 8 } }, { "memory": 9361082624, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 8, "layer.27.mlp.down_proj": 9 } }, { "memory": 9370290432, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 8, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9379498240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 8, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9379858688, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 8, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9389066496, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 8, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9398274304, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 8, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9407482112, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 8, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9416689920, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 8, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9425897728, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 8, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9435105536, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 8, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9444313344, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 8, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9444673792, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 8, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9445034240, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 8, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9454242048, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 8, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9463449856, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 8, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9472657664, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 8, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9481865472, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 8, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9491073280, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 8, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9500281088, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 8, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9509488896, "layers": { "layer.0.self_attn.q_proj": 8, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9511323904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 8, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9520531712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 8, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9529739520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 8, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9530099968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 8, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9539307776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 8, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9548515584, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 8, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9557723392, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 8, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9566931200, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 8, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9576139008, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 8, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9585346816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 8, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9594554624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 8, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9596389632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 8, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9605597440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 8, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9605957888, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 8, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9615165696, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 8, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9624373504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 8, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9626208512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 8, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9626568960, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 8, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9628403968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 8, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9630238976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 8, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9630599424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 8, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9639807232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 8, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9649015040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 8, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9650850048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 8, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9651210496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 8, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9660418304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 8, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9669626112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 8, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9678833920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 8, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9688041728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 8, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9697249536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 8, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9699084544, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 8, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9700919552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 8, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9710127360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 8, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9711962368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 8, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9721170176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 8, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9730377984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 8, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9730738432, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 8, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9732573440, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 8, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9741781248, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 8, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9750989056, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 8, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9760196864, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 8, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9760557312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 8, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9769765120, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 8, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9778972928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 8, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9788180736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 8, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9788541184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 8, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9788901632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 8, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9790736640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 8, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9791097088, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 8, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9800304896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 8, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9800665344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 8, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9802500352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 8, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9811708160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 8, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9813543168, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 8, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9822750976, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 8, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9823111424, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 8, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9832319232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 8, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9841527040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 8, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9841887488, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 8, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9842247936, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 8, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9851455744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 8, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9860663552, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 8, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9861024000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 8, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9861384448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 8, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9863219456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 8, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9865054464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 8, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9865414912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 8, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9867249920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 8, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9876457728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 8, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9878292736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 8, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9880127744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 8, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9880488192, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 8, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9880848640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 8, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9890056448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 8, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9891891456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 8, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9892251904, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 8, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9894086912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 8, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9894447360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 8, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9894807808, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 8, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9896642816, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 8, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9905850624, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 8, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9907685632, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 8, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9908046080, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 8, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9908406528, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 8, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9917614336, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 8, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9926822144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 8, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9927182592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 8, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9929017600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 8, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9929378048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 8, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9938585856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 8, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9938946304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 8, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9948154112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 8, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9957361920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 8, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9966569728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 8, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9968404736, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 8, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9970239744, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 8, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9970600192, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 8, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9979808000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 8, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9981643008, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 8, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9982003456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 8, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9983838464, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 8, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9984198912, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 8, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9984559360, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 8, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9986394368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 8, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9995602176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 8, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9997437184, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 8, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 9999272192, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 8, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10008480000, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 8, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10008840448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 8, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10010675456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 8, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10019883264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 8, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10020243712, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 8, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10020604160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 8, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10029811968, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 8, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10030172416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 8, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10039380224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 8, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10041215232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 8, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10050423040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 8, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10052258048, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 8, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10061465856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 8, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10061826304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 8, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10062186752, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 8, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10064021760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 8, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10065856768, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 8, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10067691776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 8, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10069526784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 8, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10069887232, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 8, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10079095040, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 8, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10079455488, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 8, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10081290496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 8, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10083125504, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 8, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10083485952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 8, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10083846400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 8, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10084206848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 8, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10084567296, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 8, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10086402304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 8, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10088237312, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 8, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10090072320, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 8, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10091907328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 8, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10092267776, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 8, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10094102784, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 8, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10103310592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 8, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10105145600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 8, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10106980608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 8, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10116188416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 8, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10125396224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 8, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10125756672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 8, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10134964480, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 8, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10135324928, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 8, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10135685376, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 8, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10136045824, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 8, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10137880832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 8, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10139715840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 8, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10141550848, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 8, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10143385856, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 8, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10152593664, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 8, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10152954112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 8, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10153314560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 8, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10153675008, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 8, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10155510016, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 8, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10157345024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 8, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10166552832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 8, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10175760640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 8, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10184968448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 8, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10186803456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 8, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10196011264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 8, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10205219072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 8, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10214426880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 8, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10223634688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 8, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10223995136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 8, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10225830144, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 8, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10226190592, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 8, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10235398400, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 8, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10244606208, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 8, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10246441216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 8, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10248276224, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 9 } }, { "memory": 10257484032, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 9, "layer.27.mlp.down_proj": 10 } }, { "memory": 10266691840, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 9, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10275899648, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 9, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10285107456, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 9, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10294315264, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 9, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10303523072, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 9, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10312730880, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 9, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10321938688, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 9, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10331146496, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 9, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10340354304, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 9, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10349562112, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 9, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10349922560, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 9, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10359130368, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 9, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10368338176, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 9, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10377545984, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 9, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10386753792, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 9, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10395961600, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 9, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10405169408, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 9, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10414377216, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 9, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10423585024, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 9, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10423945472, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 9, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10424305920, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 9, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10433513728, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 9, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10442721536, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 9, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10451929344, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 9, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10453764352, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 9, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10462972160, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 9, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10463332608, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 9, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10472540416, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 9, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10472900864, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 9, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10482108672, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 9, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10483943680, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 9, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10484304128, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 9, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10484664576, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 9, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10493872384, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 9, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10494232832, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 9, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10503440640, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 9, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10512648448, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 9, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10513008896, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 9, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10522216704, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 9, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10531424512, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 9, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10533259520, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 9, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10542467328, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 9, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10551675136, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 9, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10560882944, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 9, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10562717952, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 9, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10571925760, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 9, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10581133568, "layers": { "layer.0.self_attn.q_proj": 9, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10582968576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 9, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10592176384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 9, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10601384192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 9, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10601744640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 9, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10602105088, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 9, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10602465536, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 9, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10602825984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 9, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10604660992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 9, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10613868800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 9, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10615703808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 9, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10624911616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 9, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10634119424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 9, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10643327232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 9, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10652535040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 9, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10652895488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 9, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10662103296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 9, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10671311104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 9, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10680518912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 9, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10680879360, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 9, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10682714368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 9, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10691922176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 9, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10693757184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 9, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10702964992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 9, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10703325440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 9, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10703685888, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 9, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10712893696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 9, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10722101504, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 9, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10723936512, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 9, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10733144320, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 9, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10733504768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 9, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10742712576, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 9, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10744547584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 9, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10753755392, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 9, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10754115840, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 9, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10755950848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 9, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10765158656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 9, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10766993664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 9, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10776201472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 9, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10785409280, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 9, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10785769728, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 9, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10787604736, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 9, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10796812544, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 9, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10797172992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 9, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10799008000, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 9, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10800843008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 9, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10801203456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 9, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10801563904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 9, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10810771712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 9, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10811132160, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 9, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10811492608, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 9, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10813327616, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 9, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10822535424, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 9, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10831743232, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 9, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10832103680, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 9, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10841311488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 9, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10850519296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 9, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10852354304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 9, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10852714752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 9, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10854549760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 9, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10854910208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 9, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10856745216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 9, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10865953024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 9, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10867788032, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 9, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10869623040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 9, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10869983488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 9, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10870343936, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 9, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10870704384, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 9, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10879912192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 9, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10881747200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 9, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10882107648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 9, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10882468096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 9, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10884303104, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 9, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10884663552, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 9, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10893871360, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 9, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10903079168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 9, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10904914176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 9, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10914121984, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 9, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10915956992, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 9, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10925164800, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 9, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10926999808, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 9, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10927360256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 9, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10936568064, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 9, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10945775872, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 9, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10947610880, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 9, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10956818688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 9, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10958653696, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 9, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10959014144, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 9, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10959374592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 9, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10959735040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 9, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10961570048, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 9, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10961930496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 9, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10971138304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 9, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10972973312, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 9, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10973333760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 9, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10975168768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 9, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10977003776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 9, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10978838784, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 9, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10988046592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 9, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10997254400, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 9, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10997614848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 9, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 10999449856, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 9, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11008657664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 9, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11017865472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 9, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11019700480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 9, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11021535488, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 9, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11023370496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 9, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11032578304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 9, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11032938752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 9, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11033299200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 9, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11042507008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 9, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11044342016, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 9, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11046177024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 9, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11046537472, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 9, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11046897920, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 9, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11047258368, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 9, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11049093376, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 9, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11058301184, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 9, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11060136192, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 9, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11061971200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 9, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11063806208, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 9, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11064166656, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 9, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11066001664, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 9, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11067836672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 9, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11068197120, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 9, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11070032128, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 9, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11071867136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 9, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11081074944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 9, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11082909952, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 9, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11084744960, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 9, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11093952768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 9, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11095787776, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 9, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11104995584, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 9, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11106830592, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 9, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11107191040, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 9, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11116398848, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 9, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11116759296, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 9, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11117119744, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 9, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11118954752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 9, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11119315200, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 9, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11119675648, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 9, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11128883456, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 9, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11129243904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 9, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11131078912, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 9, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11131439360, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 9, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11140647168, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 9, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11142482176, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 9, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11142842624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 9, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11144677632, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 10 } }, { "memory": 11153885440, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 10, "layer.27.mlp.down_proj": 11 } }, { "memory": 11163093248, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 10, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11172301056, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 10, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11181508864, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 10, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11190716672, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 10, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11199924480, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 10, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11209132288, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 10, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11218340096, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 10, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11227547904, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 10, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11236755712, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 10, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11245963520, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 10, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11255171328, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 10, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11264379136, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 10, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11273586944, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 10, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11282794752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 10, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11292002560, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 10, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11292363008, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 10, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11301570816, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 10, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11310778624, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 10, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11319986432, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 10, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11329194240, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 10, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11329554688, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 10, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11338762496, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 10, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11347970304, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 10, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11348330752, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 10, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11350165760, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 10, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11352000768, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 10, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11352361216, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 10, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11361569024, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 10, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11370776832, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 10, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11379984640, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 10, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11389192448, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 10, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11398400256, "layers": { "layer.0.self_attn.q_proj": 10, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11400235264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 10, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11400595712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 10, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11409803520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 10, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11419011328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 10, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11428219136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 10, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11437426944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 10, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11437787392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 10, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11446995200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 10, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11447355648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 10, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11449190656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 10, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11451025664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 10, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11452860672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 10, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11462068480, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 10, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11462428928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 10, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11462789376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 10, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11471997184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 10, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11472357632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 10, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11474192640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 10, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11476027648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 10, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11485235456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 10, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11487070464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 10, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11488905472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 10, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11489265920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 10, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11491100928, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 10, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11492935936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 10, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11493296384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 10, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11493656832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 10, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11502864640, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 10, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11512072448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 10, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11521280256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 10, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11530488064, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 10, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11530848512, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 10, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11531208960, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 10, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11540416768, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 10, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11549624576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 10, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11551459584, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 10, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11560667392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 10, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11569875200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 10, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11579083008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 10, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11588290816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 10, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11588651264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 10, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11597859072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 10, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11607066880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 10, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11616274688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 10, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11625482496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 10, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11625842944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 10, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11635050752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 10, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11644258560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 10, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11644619008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 10, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11646454016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 10, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11648289024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 10, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11650124032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 10, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11659331840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 10, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11661166848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 10, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11663001856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 10, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11672209664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 10, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11674044672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 10, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11675879680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 10, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11677714688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 10, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11686922496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 10, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11696130304, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 10, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11697965312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 10, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11707173120, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 10, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11709008128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 10, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11718215936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 10, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11718576384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 10, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11718936832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 10, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11719297280, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 10, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11728505088, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 10, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11730340096, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 10, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11739547904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 10, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11739908352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 10, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11740268800, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 10, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11749476608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 10, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11749837056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 10, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11759044864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 10, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11760879872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 10, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11770087680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 10, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11770448128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 10, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11770808576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 10, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11771169024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 10, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11780376832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 10, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11782211840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 10, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11791419648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 10, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11793254656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 10, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11795089664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 10, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11796924672, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 10, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11798759680, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 10, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11799120128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 10, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11808327936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 10, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11817535744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 10, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11819370752, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 10, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11819731200, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 10, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11821566208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 10, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11823401216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 10, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11823761664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 10, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11824122112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 10, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11824482560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 10, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11826317568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 10, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11828152576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 10, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11828513024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 10, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11837720832, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 10, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11839555840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 10, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11841390848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 10, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11841751296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 10, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11842111744, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 10, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11842472192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 10, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11851680000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 10, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11852040448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 10, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11853875456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 10, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11854235904, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 10, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11854596352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 10, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11856431360, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 10, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11856791808, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 10, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11857152256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 10, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11857512704, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 10, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11859347712, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 10, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11859708160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 10, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11860068608, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 10, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11869276416, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 10, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11878484224, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 10, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11880319232, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 10, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11882154240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 10, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11891362048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 10, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11900569856, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 10, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11902404864, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 10, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11904239872, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 10, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11906074880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 10, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11907909888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 10, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11917117696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 10, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11917478144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 10, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11919313152, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 10, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11921148160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 10, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11922983168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 10, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11924818176, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 10, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11926653184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 10, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11927013632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 10, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11936221440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 10, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11936581888, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 10, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11936942336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 10, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11937302784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 10, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11939137792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 10, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11939498240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 10, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11939858688, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 10, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11941693696, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 10, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11950901504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 10, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11960109312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 10, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11960469760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 10, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11960830208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 10, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11970038016, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 10, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11970398464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 10, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11972233472, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 10, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11972593920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 10, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11981801728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 10, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11983636736, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 10, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 11992844544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 10, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12002052352, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 10, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12011260160, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 10, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12013095168, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 10, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12022302976, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 10, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12031510784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 10, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12040718592, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 10, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12041079040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 11 } }, { "memory": 12050286848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 11, "layer.27.mlp.down_proj": 12 } }, { "memory": 12059494656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 11, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12068702464, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 11, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12077910272, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 11, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12087118080, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 11, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12087478528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 11, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12096686336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 11, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12105894144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 11, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12115101952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 11, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12124309760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 11, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12133517568, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 11, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12142725376, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 11, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12151933184, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 11, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12152293632, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 11, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12161501440, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 11, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12170709248, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 11, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12179917056, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 11, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12180277504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 11, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12189485312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 11, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12191320320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 11, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12200528128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 11, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12200888576, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 11, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12210096384, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 11, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12219304192, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 11, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12228512000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 11, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12228872448, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 11, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12238080256, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 11, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12239915264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 11, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12249123072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 11, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12249483520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 11, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12249843968, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 11, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12259051776, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 11, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12260886784, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 11, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12262721792, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 11, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12263082240, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 11, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12272290048, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 11, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12272650496, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 11, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12274485504, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 11, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12283693312, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 11, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12285528320, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 11, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12294736128, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 11, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12303943936, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 11, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12305778944, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 11, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12306139392, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 11, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12306499840, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 11, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12315707648, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 11, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12324915456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 11, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12334123264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 11, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12343331072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 11, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12352538880, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 11, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12352899328, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 11, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12362107136, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 11, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12363942144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 11, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12373149952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 11, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12382357760, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 11, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12382718208, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 11, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12384553216, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 11, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12393761024, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 11, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12395596032, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 11, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12397431040, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 11, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12406638848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 11, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12406999296, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 11, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12416207104, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 11, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12418042112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 11, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12418402560, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 11, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12418763008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 11, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12419123456, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 11, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12428331264, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 11, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12437539072, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 11, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12437899520, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 11, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12439734528, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 11, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12448942336, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 11, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12458150144, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 11, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12467357952, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 11, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12467718400, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 11, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12468078848, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 11, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12477286656, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 11, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12479121664, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 11, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12479482112, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 11, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12488689920, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 11, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12497897728, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 11, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12507105536, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 11, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12508940544, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 11, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12509300992, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 11, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12511136000, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 11, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12512971008, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 11, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12522178816, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 11, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12531386624, "layers": { "layer.0.self_attn.q_proj": 11, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12533221632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 11, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12542429440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 11, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12542789888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 11, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12544624896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 11, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12553832704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 11, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12554193152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 11, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12554553600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 11, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12563761408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 11, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12564121856, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 11, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12573329664, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 11, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12573690112, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 11, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12582897920, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 11, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12584732928, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 11, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12586567936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 11, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12595775744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 11, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12604983552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 11, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12605344000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 11, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12605704448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 11, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12607539456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 11, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12607899904, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 11, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12617107712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 11, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12618942720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 11, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12619303168, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 11, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12619663616, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 11, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12620024064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 11, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12621859072, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 11, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12623694080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 11, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12632901888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 11, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12633262336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 11, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12633622784, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 11, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12635457792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 11, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12644665600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 11, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12653873408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 11, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12655708416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 11, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12664916224, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 11, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12665276672, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 11, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12667111680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 11, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12668946688, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 11, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12670781696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 11, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12679989504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 11, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12680349952, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 11, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12682184960, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 11, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12691392768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 11, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12693227776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 11, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12693588224, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 11, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12695423232, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 11, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12697258240, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 11, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12706466048, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 11, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12706826496, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 11, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12708661504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 11, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12710496512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 11, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12719704320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 11, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12721539328, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 11, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12730747136, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 11, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12731107584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 11, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12731468032, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 11, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12731828480, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 11, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12732188928, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 11, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12741396736, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 11, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12750604544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 11, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12750964992, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 11, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12752800000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 11, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12762007808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 11, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12763842816, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 11, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12765677824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 11, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12766038272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 11, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12775246080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 11, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12777081088, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 11, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12778916096, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 11, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12779276544, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 11, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12781111552, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 11, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12782946560, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 11, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12783307008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 11, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12783667456, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 11, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12785502464, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 11, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12794710272, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 11, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12795070720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 11, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12804278528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 11, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12804638976, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 11, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12806473984, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 11, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12806834432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 11, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12808669440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 11, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12809029888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 11, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12810864896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 11, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12820072704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 11, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12821907712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 11, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12823742720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 11, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12832950528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 11, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12842158336, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 11, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12843993344, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 11, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12853201152, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 11, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12862408960, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 11, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12862769408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 11, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12864604416, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 11, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12866439424, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 11, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12868274432, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 11, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12870109440, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 11, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12870469888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 11, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12879677696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 11, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12888885504, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 11, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12890720512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 11, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12891080960, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 11, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12900288768, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 11, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12909496576, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 11, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12909857024, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 11, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12919064832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 11, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12928272640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 11, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12937480448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 12 } }, { "memory": 12946688256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 12, "layer.27.mlp.down_proj": 13 } }, { "memory": 12955896064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 12, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 12965103872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 12, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 12974311680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 12, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 12983519488, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 12, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 12992727296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 12, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13001935104, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 12, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13011142912, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 12, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13020350720, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 12, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13029558528, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 12, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13031393536, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 12, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13031753984, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 12, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13040961792, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 12, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13050169600, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 12, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13059377408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 12, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13068585216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 12, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13077793024, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 12, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13087000832, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 12, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13096208640, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 12, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13105416448, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 12, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13105776896, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 12, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13114984704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 12, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13124192512, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 12, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13133400320, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 12, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13142608128, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 12, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13151815936, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 12, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13161023744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 12, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13161384192, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 12, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13170592000, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 12, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13179799808, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 12, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13180160256, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 12, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13189368064, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 12, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13198575872, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 12, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13207783680, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 12, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13216991488, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 12, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13226199296, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 12, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13226559744, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 12, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13228394752, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 12, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13228755200, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 12, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13237963008, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 12, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13239798016, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 12, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13249005824, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 12, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13258213632, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 12, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13258574080, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 12, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13267781888, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 12, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13276989696, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 12, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13278824704, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 12, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13280659712, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 12, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13281020160, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 12, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13290227968, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 12, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13299435776, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 12, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13308643584, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 12, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13317851392, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 12, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13319686400, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 12, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13321521408, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 12, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13330729216, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 12, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13339937024, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 12, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13340297472, "layers": { "layer.0.self_attn.q_proj": 12, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13342132480, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 12, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13343967488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 12, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13345802496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 12, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13346162944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 12, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13346523392, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 12, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13348358400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 12, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13350193408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 12, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13359401216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 12, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13359761664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 12, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13368969472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 12, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13369329920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 12, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13371164928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 12, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13371525376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 12, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13373360384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 12, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13373720832, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 12, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13375555840, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 12, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13384763648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 12, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13386598656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 12, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13395806464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 12, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13396166912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 12, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13398001920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 12, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13399836928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 12, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13409044736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 12, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13410879744, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 12, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13411240192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 12, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13411600640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 12, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13413435648, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 12, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13415270656, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 12, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13424478464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 12, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13433686272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 12, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13434046720, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 12, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13435881728, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 12, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13437716736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 12, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13446924544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 12, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13448759552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 12, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13450594560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 12, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13450955008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 12, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13460162816, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 12, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13469370624, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 12, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13471205632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 12, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13471566080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 12, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13480773888, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 12, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13482608896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 12, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13482969344, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 12, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13483329792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 12, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13485164800, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 12, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13485525248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 12, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13494733056, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 12, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13495093504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 12, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13495453952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 12, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13495814400, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 12, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13496174848, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 12, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13496535296, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 12, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13498370304, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 12, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13498730752, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 12, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13507938560, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 12, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13508299008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 12, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13508659456, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 12, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13510494464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 12, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13519702272, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 12, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13528910080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 12, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13529270528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 12, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13531105536, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 12, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13532940544, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 12, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13534775552, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 12, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13535136000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 12, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13536971008, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 12, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13538806016, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 12, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13539166464, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 12, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13539526912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 12, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13541361920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 12, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13543196928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 12, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13543557376, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 12, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13543917824, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 12, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13553125632, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 12, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13553486080, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 12, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13553846528, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 12, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13563054336, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 12, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13572262144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 12, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13574097152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 12, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13574457600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 12, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13583665408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 12, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13585500416, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 12, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13585860864, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 12, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13587695872, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 12, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13596903680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 12, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13598738688, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 12, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13607946496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 12, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13609781504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 12, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13618989312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 12, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13620824320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 12, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13622659328, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 12, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13631867136, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 12, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13633702144, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 12, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13635537152, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 12, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13635897600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 12, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13637732608, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 12, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13639567616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 12, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13648775424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 12, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13657983232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 12, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13667191040, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 12, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13667551488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 12, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13669386496, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 12, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13669746944, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 12, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13671581952, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 12, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13673416960, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 12, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13673777408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 12, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13682985216, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 12, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13683345664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 12, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13683706112, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 12, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13692913920, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 12, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13694748928, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 12, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13703956736, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 12, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13704317184, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 12, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13713524992, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 12, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13713885440, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 12, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13723093248, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 12, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13723453696, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 12, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13732661504, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 12, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13741869312, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 12, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13751077120, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 12, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13752912128, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 12, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13762119936, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 12, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13762480384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 12, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13771688192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 12, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13772048640, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 12, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13772409088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 12, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13781616896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 12, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13783451904, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 12, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13785286912, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 12, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13785647360, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 12, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13787482368, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 12, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13796690176, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 12, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13805897984, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 12, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13815105792, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 12, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13824313600, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 12, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13833521408, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 12, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13833881856, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 13 } }, { "memory": 13843089664, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 13, "layer.27.mlp.down_proj": 14 } }, { "memory": 13852297472, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 13, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13861505280, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 13, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13870713088, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 13, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13879920896, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 13, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13889128704, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 13, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13898336512, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 13, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13907544320, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 13, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13907904768, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 13, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13917112576, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 13, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13926320384, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 13, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13935528192, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 13, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13944736000, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 13, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13953943808, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 13, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13963151616, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 13, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13972359424, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 13, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13981567232, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 13, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13981927680, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 13, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13991135488, "layers": { "layer.0.self_attn.q_proj": 13, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 13992970496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 13, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14002178304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 13, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14002538752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 13, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14002899200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 13, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14012107008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 13, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14021314816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 13, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14030522624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 13, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14030883072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 13, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14032718080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 13, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14034553088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 13, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14043760896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 13, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14052968704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 13, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14054803712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 13, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14064011520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 13, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14065846528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 13, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14067681536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 13, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14076889344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 13, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14078724352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 13, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14079084800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 13, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14088292608, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 13, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14088653056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 13, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14097860864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 13, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14107068672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 13, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14108903680, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 13, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14110738688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 13, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14119946496, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 13, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14129154304, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 13, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14130989312, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 13, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14131349760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 13, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14131710208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 13, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14132070656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 13, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14141278464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 13, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14150486272, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 13, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14150846720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 13, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14160054528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 13, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14160414976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 13, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14160775424, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 13, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14169983232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 13, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14179191040, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 13, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14179551488, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 13, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14188759296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 13, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14189119744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 13, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14198327552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 13, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14207535360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 13, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14209370368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 13, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14211205376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 13, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14211565824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 13, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14213400832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 13, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14222608640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 13, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14224443648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 13, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14226278656, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 13, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14235486464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 13, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14235846912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 13, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14236207360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 13, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14238042368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 13, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14239877376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 13, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14249085184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 13, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14258292992, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 13, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14260128000, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 13, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14260488448, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 13, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14260848896, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 13, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14262683904, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 13, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14264518912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 13, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14266353920, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 13, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14275561728, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 13, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14284769536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 13, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14285129984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 13, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14294337792, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 13, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14296172800, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 13, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14296533248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 13, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14305741056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 13, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14314948864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 13, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14316783872, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 13, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14317144320, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 13, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14317504768, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 13, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14326712576, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 13, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14328547584, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 13, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14328908032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 13, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14338115840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 13, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14338476288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 13, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14338836736, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 13, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14340671744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 13, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14349879552, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 13, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14359087360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 13, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14368295168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 13, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14368655616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 13, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14369016064, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 13, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14370851072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 13, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14371211520, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 13, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14373046528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 13, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14373406976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 13, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14382614784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 13, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14382975232, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 13, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14384810240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 13, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14394018048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 13, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14395853056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 13, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14405060864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 13, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14414268672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 13, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14414629120, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 13, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14414989568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 13, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14415350016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 13, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14415710464, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 13, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14416070912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 13, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14416431360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 13, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14418266368, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 13, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14427474176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 13, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14427834624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 13, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14429669632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 13, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14430030080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 13, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14430390528, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 13, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14432225536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 13, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14441433344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 13, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14443268352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 13, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14452476160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 13, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14454311168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 13, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14456146176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 13, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14457981184, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 13, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14458341632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 13, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14467549440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 13, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14476757248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 13, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14477117696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 13, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14478952704, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 13, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14480787712, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 13, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14481148160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 13, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14482983168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 13, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14483343616, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 13, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14485178624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 13, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14485539072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 13, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14494746880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 13, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14495107328, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 13, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14496942336, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 13, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14498777344, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 13, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14507985152, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 13, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14509820160, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 13, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14511655168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 13, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14513490176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 13, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14522697984, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 13, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14523058432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 13, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14524893440, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 13, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14534101248, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 13, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14534461696, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 13, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14543669504, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 13, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14544029952, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 13, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14544390400, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 13, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14546225408, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 13, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14548060416, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 13, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14557268224, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 13, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14566476032, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 13, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14575683840, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 13, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14576044288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 13, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14577879296, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 13, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14578239744, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 13, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14580074752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 13, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14581909760, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 13, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14591117568, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 13, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14600325376, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 13, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14600685824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 13, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14609893632, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 13, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14610254080, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 13, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14612089088, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 13, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14612449536, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 13, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14614284544, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 13, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14623492352, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 13, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14625327360, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 13, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14634535168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 13, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14643742976, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 13, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14652950784, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 13, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14662158592, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 13, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14671366400, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 13, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14680574208, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 13, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14689782016, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 13, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14698989824, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 13, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14700824832, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 13, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14710032640, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 13, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14711867648, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 13, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14721075456, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 13, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14730283264, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 14 } }, { "memory": 14739491072, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 14, "layer.27.mlp.down_proj": 15 } }, { "memory": 14748698880, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 14, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14757906688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 14, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14758267136, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 14, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14767474944, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 14, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14776682752, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 14, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14777043200, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 14, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14786251008, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 14, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14795458816, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 14, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14804666624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 14, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14813874432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 14, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14823082240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 14, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14832290048, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 14, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14834125056, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 14, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14843332864, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 14, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14852540672, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 14, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14861748480, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 14, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14870956288, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 14, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14880164096, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 14, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14881999104, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 14, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14891206912, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 14, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14900414720, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 14, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14900775168, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 14, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14902610176, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 14, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14902970624, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 14, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14912178432, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 14, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14921386240, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 14, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14921746688, "layers": { "layer.0.self_attn.q_proj": 14, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14923581696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 14, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14932789504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 14, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14941997312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 14, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14951205120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 14, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14960412928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 14, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14969620736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 14, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14978828544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 14, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14988036352, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 14, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 14997244160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 14, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15006451968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 14, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15015659776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 14, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15017494784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 14, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15026702592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 14, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15027063040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 14, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15028898048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 14, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15029258496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 14, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15031093504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 14, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15032928512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 14, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15033288960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 14, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15042496768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 14, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15044331776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 14, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15053539584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 14, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15062747392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 14, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15063107840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 14, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15063468288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 14, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15065303296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 14, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15067138304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 14, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15076346112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 14, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15085553920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 14, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15087388928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 14, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15087749376, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 14, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15089584384, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 14, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15098792192, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 14, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15099152640, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 14, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15108360448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 14, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15108720896, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 14, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15110555904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 14, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15119763712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 14, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15120124160, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 14, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15120484608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 14, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15122319616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 14, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15131527424, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 14, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15133362432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 14, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15142570240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 14, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15151778048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 14, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15153613056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 14, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15162820864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 14, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15172028672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 14, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15172389120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 14, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15174224128, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 14, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15174584576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 14, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15176419584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 14, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15185627392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 14, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15187462400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 14, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15187822848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 14, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15189657856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 14, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15190018304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 14, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15191853312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 14, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15192213760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 14, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15192574208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 14, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15192934656, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 14, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15202142464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 14, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15202502912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 14, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15211710720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 14, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15212071168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 14, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15212431616, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 14, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15212792064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 14, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15221999872, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 14, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15223834880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 14, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15225669888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 14, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15226030336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 14, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15227865344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 14, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15237073152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 14, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15237433600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 14, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15239268608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 14, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15248476416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 14, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15248836864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 14, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15249197312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 14, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15249557760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 14, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15251392768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 14, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15251753216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 14, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15260961024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 14, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15262796032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 14, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15263156480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 14, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15264991488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 14, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15265351936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 14, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15267186944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 14, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15269021952, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 14, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15278229760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 14, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15280064768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 14, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15289272576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 14, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15289633024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 14, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15298840832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 14, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15300675840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 14, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15309883648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 14, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15310244096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 14, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15312079104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 14, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15321286912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 14, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15323121920, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 14, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15324956928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 14, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15334164736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 14, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15334525184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 14, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15343732992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 14, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15345568000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 14, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15347403008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 14, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15347763456, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 14, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15356971264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 14, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15357331712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 14, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15359166720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 14, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15359527168, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 14, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15361362176, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 14, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15370569984, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 14, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15370930432, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 14, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15380138240, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 14, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15381973248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 14, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15383808256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 14, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15385643264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 14, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15394851072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 14, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15395211520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 14, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15395571968, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 14, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15404779776, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 14, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15406614784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 14, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15406975232, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 14, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15416183040, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 14, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15418018048, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 14, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15419853056, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 14, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15429060864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 14, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15438268672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 14, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15447476480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 14, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15447836928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 14, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15457044736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 14, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15466252544, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 14, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15468087552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 14, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15468448000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 14, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15470283008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 14, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15479490816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 14, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15479851264, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 14, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15489059072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 14, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15489419520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 14, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15491254528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 14, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15500462336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 14, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15500822784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 14, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15510030592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 14, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15519238400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 14, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15519598848, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 14, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15521433856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 14, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15523268864, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 14, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15532476672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 14, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15541684480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 14, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15542044928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 14, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15543879936, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 14, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15545714944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 14, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15546075392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 14, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15547910400, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 14, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15557118208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 14, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15566326016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 14, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15566686464, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 14, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15575894272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 14, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15585102080, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 14, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15594309888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 14, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15603517696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 14, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15603878144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 14, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15604238592, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 14, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15606073600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 14, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15607908608, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 14, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15617116416, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 14, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15626324224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 14, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15626684672, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 15 } }, { "memory": 15635892480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 15, "layer.27.mlp.down_proj": 16 } }, { "memory": 15645100288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 15, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15654308096, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 15, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15663515904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 15, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15672723712, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 15, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15681931520, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 15, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15691139328, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 15, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15700347136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 15, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15709554944, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 15, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15718762752, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 15, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15727970560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 15, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15737178368, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 15, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15737538816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 15, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15746746624, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 15, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15747107072, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 15, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15756314880, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 15, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15756675328, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 15, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15765883136, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 15, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15766243584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 15, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15775451392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 15, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15784659200, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 15, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15793867008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 15, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15803074816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 15, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15804909824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 15, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15814117632, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 15, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15814478080, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 15, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15823685888, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 15, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15832893696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 15, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15833254144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 15, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15842461952, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 15, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15851669760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 15, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15852030208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 15, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15861238016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 15, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15870445824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 15, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15872280832, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 15, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15874115840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 15, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15883323648, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 15, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15892531456, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 15, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15892891904, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 15, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15894726912, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 15, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15903934720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 15, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15913142528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 15, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15922350336, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 15, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15922710784, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 15, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15924545792, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 15, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15926380800, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 15, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15926741248, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 15, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15927101696, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 15, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15927462144, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 15, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15936669952, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 15, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15938504960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 15, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15938865408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 15, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15948073216, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 15, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15949908224, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 15, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15959116032, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 15, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15968323840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 15, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15968684288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 15, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15970519296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 15, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15979727104, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 15, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15981562112, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 15, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15981922560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 15, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15982283008, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 15, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15991490816, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 15, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15993325824, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 15, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15993686272, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 15, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 15994046720, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 15, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16003254528, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 15, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16005089536, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 15, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16014297344, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 15, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16023505152, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 15, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16023865600, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 15, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16033073408, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 15, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16033433856, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 15, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16033794304, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 15, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16035629312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 15, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16044837120, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 15, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16054044928, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 15, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16063252736, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 15, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16063613184, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 15, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16072820992, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 15, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16074656000, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 15, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16075016448, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 15, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16084224256, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 15, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16093432064, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 15, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16093792512, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 15, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16094152960, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 15, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16103360768, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 15, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16112568576, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 15, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16114403584, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 15, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16123611392, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 15, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16123971840, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 15, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16124332288, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 15, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16126167296, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 15, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16126527744, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 15, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16135735552, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 15, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16137570560, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 15, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16139405568, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 15, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16139766016, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 15, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16141601024, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 15, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16141961472, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 15, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16143796480, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 15, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16145631488, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 15, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16147466496, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 15, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16149301504, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 15, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16158509312, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 15, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16158869760, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 15, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16159230208, "layers": { "layer.0.self_attn.q_proj": 15, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16161065216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 15, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16161425664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 15, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16163260672, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 15, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16172468480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 15, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16181676288, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 15, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16190884096, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 15, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16191244544, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 15, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16193079552, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 15, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16193440000, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 15, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16193800448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 15, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16195635456, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 15, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16195995904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 15, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16197830912, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 15, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16207038720, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 15, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16207399168, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 15, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16209234176, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 15, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16211069184, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 15, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16220276992, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 15, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16220637440, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 15, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16222472448, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 15, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16231680256, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 15, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16233515264, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 15, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16235350272, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 15, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16244558080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 15, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16244918528, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 15, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16254126336, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 15, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16263334144, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 15, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16272541952, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 15, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16272902400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 15, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16274737408, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 15, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16283945216, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 15, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16284305664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 15, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16284666112, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 15, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16285026560, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 15, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16294234368, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 15, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16303442176, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 15, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16312649984, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 15, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16321857792, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 15, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16323692800, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 15, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16332900608, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 15, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16334735616, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 15, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16336570624, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 15, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16336931072, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 15, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16346138880, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 15, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16347973888, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 15, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16349808896, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 15, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16351643904, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 15, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16352004352, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 15, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16361212160, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 15, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16370419968, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 15, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16370780416, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 15, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16372615424, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 15, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16381823232, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 15, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16391031040, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 15, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16392866048, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 15, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16402073856, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 15, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16411281664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 15, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16420489472, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 15, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16422324480, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 15, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16422684928, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 15, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16431892736, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 15, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16433727744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 15, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16434088192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 15, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16435923200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 15, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16436283648, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 15, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16438118656, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 15, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16439953664, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 15, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16441788672, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 15, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16443623680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 15, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16443984128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 15, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16444344576, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 15, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16453552384, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 15, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16455387392, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 15, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16457222400, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 15, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16457582848, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 15, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16459417856, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 15, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16461252864, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 15, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16463087872, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 15, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16472295680, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 15, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16472656128, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 15, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16481863936, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 15, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16491071744, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 15, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16491432192, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 15, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16493267200, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 15, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16495102208, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 15, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16504310016, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 15, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16513517824, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 15, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16522725632, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 15, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } }, { "memory": 16523086080, "layers": { "layer.0.self_attn.q_proj": 16, "layer.0.self_attn.k_proj": 16, "layer.0.self_attn.v_proj": 16, "layer.0.self_attn.o_proj": 16, "layer.0.mlp.gate_proj": 16, "layer.0.mlp.up_proj": 16, "layer.0.mlp.down_proj": 16, "layer.1.self_attn.q_proj": 16, "layer.1.self_attn.k_proj": 16, "layer.1.self_attn.v_proj": 16, "layer.1.self_attn.o_proj": 16, "layer.1.mlp.gate_proj": 16, "layer.1.mlp.up_proj": 16, "layer.1.mlp.down_proj": 16, "layer.2.self_attn.q_proj": 16, "layer.2.self_attn.k_proj": 16, "layer.2.self_attn.v_proj": 16, "layer.2.self_attn.o_proj": 16, "layer.2.mlp.gate_proj": 16, "layer.2.mlp.up_proj": 16, "layer.2.mlp.down_proj": 16, "layer.3.self_attn.q_proj": 16, "layer.3.self_attn.k_proj": 16, "layer.3.self_attn.v_proj": 16, "layer.3.self_attn.o_proj": 16, "layer.3.mlp.gate_proj": 16, "layer.3.mlp.up_proj": 16, "layer.3.mlp.down_proj": 16, "layer.4.self_attn.q_proj": 16, "layer.4.self_attn.k_proj": 16, "layer.4.self_attn.v_proj": 16, "layer.4.self_attn.o_proj": 16, "layer.4.mlp.gate_proj": 16, "layer.4.mlp.up_proj": 16, "layer.4.mlp.down_proj": 16, "layer.5.self_attn.q_proj": 16, "layer.5.self_attn.k_proj": 16, "layer.5.self_attn.v_proj": 16, "layer.5.self_attn.o_proj": 16, "layer.5.mlp.gate_proj": 16, "layer.5.mlp.up_proj": 16, "layer.5.mlp.down_proj": 16, "layer.6.self_attn.q_proj": 16, "layer.6.self_attn.k_proj": 16, "layer.6.self_attn.v_proj": 16, "layer.6.self_attn.o_proj": 16, "layer.6.mlp.gate_proj": 16, "layer.6.mlp.up_proj": 16, "layer.6.mlp.down_proj": 16, "layer.7.self_attn.q_proj": 16, "layer.7.self_attn.k_proj": 16, "layer.7.self_attn.v_proj": 16, "layer.7.self_attn.o_proj": 16, "layer.7.mlp.gate_proj": 16, "layer.7.mlp.up_proj": 16, "layer.7.mlp.down_proj": 16, "layer.8.self_attn.q_proj": 16, "layer.8.self_attn.k_proj": 16, "layer.8.self_attn.v_proj": 16, "layer.8.self_attn.o_proj": 16, "layer.8.mlp.gate_proj": 16, "layer.8.mlp.up_proj": 16, "layer.8.mlp.down_proj": 16, "layer.9.self_attn.q_proj": 16, "layer.9.self_attn.k_proj": 16, "layer.9.self_attn.v_proj": 16, "layer.9.self_attn.o_proj": 16, "layer.9.mlp.gate_proj": 16, "layer.9.mlp.up_proj": 16, "layer.9.mlp.down_proj": 16, "layer.10.self_attn.q_proj": 16, "layer.10.self_attn.k_proj": 16, "layer.10.self_attn.v_proj": 16, "layer.10.self_attn.o_proj": 16, "layer.10.mlp.gate_proj": 16, "layer.10.mlp.up_proj": 16, "layer.10.mlp.down_proj": 16, "layer.11.self_attn.q_proj": 16, "layer.11.self_attn.k_proj": 16, "layer.11.self_attn.v_proj": 16, "layer.11.self_attn.o_proj": 16, "layer.11.mlp.gate_proj": 16, "layer.11.mlp.up_proj": 16, "layer.11.mlp.down_proj": 16, "layer.12.self_attn.q_proj": 16, "layer.12.self_attn.k_proj": 16, "layer.12.self_attn.v_proj": 16, "layer.12.self_attn.o_proj": 16, "layer.12.mlp.gate_proj": 16, "layer.12.mlp.up_proj": 16, "layer.12.mlp.down_proj": 16, "layer.13.self_attn.q_proj": 16, "layer.13.self_attn.k_proj": 16, "layer.13.self_attn.v_proj": 16, "layer.13.self_attn.o_proj": 16, "layer.13.mlp.gate_proj": 16, "layer.13.mlp.up_proj": 16, "layer.13.mlp.down_proj": 16, "layer.14.self_attn.q_proj": 16, "layer.14.self_attn.k_proj": 16, "layer.14.self_attn.v_proj": 16, "layer.14.self_attn.o_proj": 16, "layer.14.mlp.gate_proj": 16, "layer.14.mlp.up_proj": 16, "layer.14.mlp.down_proj": 16, "layer.15.self_attn.q_proj": 16, "layer.15.self_attn.k_proj": 16, "layer.15.self_attn.v_proj": 16, "layer.15.self_attn.o_proj": 16, "layer.15.mlp.gate_proj": 16, "layer.15.mlp.up_proj": 16, "layer.15.mlp.down_proj": 16, "layer.16.self_attn.q_proj": 16, "layer.16.self_attn.k_proj": 16, "layer.16.self_attn.v_proj": 16, "layer.16.self_attn.o_proj": 16, "layer.16.mlp.gate_proj": 16, "layer.16.mlp.up_proj": 16, "layer.16.mlp.down_proj": 16, "layer.17.self_attn.q_proj": 16, "layer.17.self_attn.k_proj": 16, "layer.17.self_attn.v_proj": 16, "layer.17.self_attn.o_proj": 16, "layer.17.mlp.gate_proj": 16, "layer.17.mlp.up_proj": 16, "layer.17.mlp.down_proj": 16, "layer.18.self_attn.q_proj": 16, "layer.18.self_attn.k_proj": 16, "layer.18.self_attn.v_proj": 16, "layer.18.self_attn.o_proj": 16, "layer.18.mlp.gate_proj": 16, "layer.18.mlp.up_proj": 16, "layer.18.mlp.down_proj": 16, "layer.19.self_attn.q_proj": 16, "layer.19.self_attn.k_proj": 16, "layer.19.self_attn.v_proj": 16, "layer.19.self_attn.o_proj": 16, "layer.19.mlp.gate_proj": 16, "layer.19.mlp.up_proj": 16, "layer.19.mlp.down_proj": 16, "layer.20.self_attn.q_proj": 16, "layer.20.self_attn.k_proj": 16, "layer.20.self_attn.v_proj": 16, "layer.20.self_attn.o_proj": 16, "layer.20.mlp.gate_proj": 16, "layer.20.mlp.up_proj": 16, "layer.20.mlp.down_proj": 16, "layer.21.self_attn.q_proj": 16, "layer.21.self_attn.k_proj": 16, "layer.21.self_attn.v_proj": 16, "layer.21.self_attn.o_proj": 16, "layer.21.mlp.gate_proj": 16, "layer.21.mlp.up_proj": 16, "layer.21.mlp.down_proj": 16, "layer.22.self_attn.q_proj": 16, "layer.22.self_attn.k_proj": 16, "layer.22.self_attn.v_proj": 16, "layer.22.self_attn.o_proj": 16, "layer.22.mlp.gate_proj": 16, "layer.22.mlp.up_proj": 16, "layer.22.mlp.down_proj": 16, "layer.23.self_attn.q_proj": 16, "layer.23.self_attn.k_proj": 16, "layer.23.self_attn.v_proj": 16, "layer.23.self_attn.o_proj": 16, "layer.23.mlp.gate_proj": 16, "layer.23.mlp.up_proj": 16, "layer.23.mlp.down_proj": 16, "layer.24.self_attn.q_proj": 16, "layer.24.self_attn.k_proj": 16, "layer.24.self_attn.v_proj": 16, "layer.24.self_attn.o_proj": 16, "layer.24.mlp.gate_proj": 16, "layer.24.mlp.up_proj": 16, "layer.24.mlp.down_proj": 16, "layer.25.self_attn.q_proj": 16, "layer.25.self_attn.k_proj": 16, "layer.25.self_attn.v_proj": 16, "layer.25.self_attn.o_proj": 16, "layer.25.mlp.gate_proj": 16, "layer.25.mlp.up_proj": 16, "layer.25.mlp.down_proj": 16, "layer.26.self_attn.q_proj": 16, "layer.26.self_attn.k_proj": 16, "layer.26.self_attn.v_proj": 16, "layer.26.self_attn.o_proj": 16, "layer.26.mlp.gate_proj": 16, "layer.26.mlp.up_proj": 16, "layer.26.mlp.down_proj": 16, "layer.27.self_attn.q_proj": 16, "layer.27.self_attn.k_proj": 16, "layer.27.self_attn.v_proj": 16, "layer.27.self_attn.o_proj": 16, "layer.27.mlp.gate_proj": 16, "layer.27.mlp.up_proj": 16, "layer.27.mlp.down_proj": 16 } } ]